{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "BERT_tokenizer.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyNCEp4XN1lRZSOEBpLxgiln",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/singularity014/BERT_sentiment_analysis_quick/blob/master/BERT_tokenizer.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sMN82Sv9xODv",
        "colab_type": "text"
      },
      "source": [
        "# I - Import Dependencies"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ulndDK8Xw54u",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import math\n",
        "import re\n",
        "import pandas as pd\n",
        "from bs4 import BeautifulSoup\n",
        "import random\n",
        "\n",
        "from google.colab import drive"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A21mIIUp0Aa_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 523
        },
        "outputId": "de6c56c1-228c-4e49-b728-e19be6a67b8a"
      },
      "source": [
        "!pip install bert-for-tf2\n",
        "!pip install sentencepiece"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting bert-for-tf2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/35/5c/6439134ecd17b33fe0396fb0b7d6ce3c5a120c42a4516ba0e9a2d6e43b25/bert-for-tf2-0.14.4.tar.gz (40kB)\n",
            "\r\u001b[K     |████████                        | 10kB 19.2MB/s eta 0:00:01\r\u001b[K     |████████████████▏               | 20kB 2.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████▎       | 30kB 2.8MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 40kB 2.1MB/s \n",
            "\u001b[?25hCollecting py-params>=0.9.6\n",
            "  Downloading https://files.pythonhosted.org/packages/a4/bf/c1c70d5315a8677310ea10a41cfc41c5970d9b37c31f9c90d4ab98021fd1/py-params-0.9.7.tar.gz\n",
            "Collecting params-flow>=0.8.0\n",
            "  Downloading https://files.pythonhosted.org/packages/a9/95/ff49f5ebd501f142a6f0aaf42bcfd1c192dc54909d1d9eb84ab031d46056/params-flow-0.8.2.tar.gz\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from params-flow>=0.8.0->bert-for-tf2) (1.18.5)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from params-flow>=0.8.0->bert-for-tf2) (4.41.1)\n",
            "Building wheels for collected packages: bert-for-tf2, py-params, params-flow\n",
            "  Building wheel for bert-for-tf2 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for bert-for-tf2: filename=bert_for_tf2-0.14.4-cp36-none-any.whl size=30114 sha256=28a9bcf589b295ad4b5037554b63b3ee6a68fa8a14dcc30f5f46ac911db8675a\n",
            "  Stored in directory: /root/.cache/pip/wheels/cf/3f/4d/79d7735015a5f523648df90d871ce8e89a7df8185f7703eeab\n",
            "  Building wheel for py-params (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for py-params: filename=py_params-0.9.7-cp36-none-any.whl size=7302 sha256=29f4881b45b75b16c4db7707543cfd21cca5ee6acd9d7d8e70722bc22940bf22\n",
            "  Stored in directory: /root/.cache/pip/wheels/67/f5/19/b461849a50aefdf4bab47c4756596e82ee2118b8278e5a1980\n",
            "  Building wheel for params-flow (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for params-flow: filename=params_flow-0.8.2-cp36-none-any.whl size=19473 sha256=48b4c19d51e4798cba0c865d2d5ccd192ae936f4636bd010b129514e085e79a3\n",
            "  Stored in directory: /root/.cache/pip/wheels/08/c8/7f/81c86b9ff2b86e2c477e3914175be03e679e596067dc630c06\n",
            "Successfully built bert-for-tf2 py-params params-flow\n",
            "Installing collected packages: py-params, params-flow, bert-for-tf2\n",
            "Successfully installed bert-for-tf2-0.14.4 params-flow-0.8.2 py-params-0.9.7\n",
            "Collecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/a4/d0a884c4300004a78cca907a6ff9a5e9fe4f090f5d95ab341c53d28cbc58/sentencepiece-0.1.91-cp36-cp36m-manylinux1_x86_64.whl (1.1MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1MB 3.4MB/s \n",
            "\u001b[?25hInstalling collected packages: sentencepiece\n",
            "Successfully installed sentencepiece-0.1.91\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cCljRz0E3e1M",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "outputId": "0a6634c3-3a37-4f23-e3e5-78004301a83d"
      },
      "source": [
        "# setting tf version to 2.x\n",
        "try:\n",
        "    %tensorflow_version 2.x\n",
        "except Exception:\n",
        "    pass\n",
        "\n",
        "import tensorflow as tf\n",
        "import tensorflow_hub as hub\n",
        "from tensorflow.keras import layers\n",
        "import bert\n",
        "tf.__version__"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic": {
              "type": "string"
            },
            "text/plain": [
              "'2.2.0'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BM8YPADU0Nzt",
        "colab_type": "text"
      },
      "source": [
        "# II - Data Preprocessing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qwUGFms124W1",
        "colab_type": "text"
      },
      "source": [
        "### Loading files\n",
        "\n",
        "\n",
        "---\n",
        "The link to the data is provided in README file of github\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rCy3DkOQ2Ncw",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 127
        },
        "outputId": "fb95c9ff-7166-41ed-88be-84cd421541e4"
      },
      "source": [
        "drive.mount(\"/content/drive\")"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cI1qr2ty6g_M",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# data cols\n",
        "cols = [\"sentiment\", \"id\", \"date\", \"query\", \"user\", \"text\"]\n",
        "# loading via google drive path\n",
        "data = pd.read_csv(\n",
        "    \"/content/drive/My Drive/sentiment_data/train_data.csv\",\n",
        "    header=None,\n",
        "    names=cols,\n",
        "    engine=\"python\",\n",
        "    encoding=\"latin1\"\n",
        ")\n",
        "\n",
        "data.drop([\"id\", \"date\", \"query\", \"user\"],\n",
        "          axis=1,\n",
        "          inplace=True)"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kVT02sBY_DPD",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 202
        },
        "outputId": "e2ac75d4-db67-42e1-8f11-cade13662239"
      },
      "source": [
        "data.head()"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sentiment</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>@switchfoot http://twitpic.com/2y1zl - Awww, t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>is upset that he can't update his Facebook by ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0</td>\n",
              "      <td>@Kenichan I dived many times for the ball. Man...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "      <td>my whole body feels itchy and like its on fire</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>@nationwideclass no, it's not behaving at all....</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   sentiment                                               text\n",
              "0          0  @switchfoot http://twitpic.com/2y1zl - Awww, t...\n",
              "1          0  is upset that he can't update his Facebook by ...\n",
              "2          0  @Kenichan I dived many times for the ball. Man...\n",
              "3          0    my whole body feels itchy and like its on fire \n",
              "4          0  @nationwideclass no, it's not behaving at all...."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YbPt3yXMFUl4",
        "colab_type": "text"
      },
      "source": [
        "### Data Cleaning"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gAM7_cdbFplR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def clean_tweet(tweet):\n",
        "    tweet = BeautifulSoup(tweet, \"lxml\").get_text()\n",
        "    # Removing the @\n",
        "    tweet = re.sub(r\"@[A-Za-z0-9]+\", ' ', tweet)\n",
        "    # Removing the URL links\n",
        "    tweet = re.sub(r\"https?://[A-Za-z0-9./]+\", ' ', tweet)\n",
        "    # Keeping only letters\n",
        "    tweet = re.sub(r\"[^a-zA-Z.!?']\", ' ', tweet)\n",
        "    # Removing additional whitespaces\n",
        "    tweet = re.sub(r\" +\", ' ', tweet)\n",
        "    return tweet\n"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7AfNqCrW7SKG",
        "colab_type": "text"
      },
      "source": [
        "#"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QJqcCcKEyK3i",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 127
        },
        "outputId": "92e9196d-4231-4511-ff30-115fc6a14e48"
      },
      "source": [
        "# clean the data\n",
        "data_clean = [clean_tweet(tweet) for tweet in data.text]\n",
        "# checking first few cleaned data\n",
        "data_clean[:5]"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[\" Awww that's a bummer. You shoulda got David Carr of Third Day to do it. D\",\n",
              " \"is upset that he can't update his Facebook by texting it... and might cry as a result School today also. Blah!\",\n",
              " ' I dived many times for the ball. Managed to save The rest go out of bounds',\n",
              " 'my whole body feels itchy and like its on fire ',\n",
              " \" no it's not behaving at all. i'm mad. why am i here? because I can't see you all over there. \"]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IRspkPF24D1E",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# changing label 4 to 1, because negative lables \n",
        "# are indicated as 4 in the dataset, hence converting them to 1\n",
        "# for 0/1 conventional label\n",
        "data_labels = data.sentiment.values\n",
        "data_labels[data_labels == 4] = 1"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t8J-K-pZ7V9K",
        "colab_type": "text"
      },
      "source": [
        "### BERT Tokenization\n",
        "\n",
        "STEP 1 - \n",
        "\n",
        "Before using BERT Tokenizer, note that we need some following info. \n",
        "*   Vocabulary details of BERT Model(path of vocab file)\n",
        "*   Whether they are lower cased or not.\n",
        "*   We will use BERT style Tokenization.\n",
        "Create a BERT layer to get both the info.\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "STEP 2 -\n",
        "\n",
        "Now that we all these info, we can create BERT tokenizer object.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jXdu1krGtVS0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# STEP 1\n",
        "bert_layer = hub.KerasLayer(\"https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1\",\n",
        "                            trainable=False)\n",
        "vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()\n",
        "do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()\n"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ww-ucPf9sV0p",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 73
        },
        "outputId": "7e2bbacd-5bcc-47ac-aa9b-5fed8656972f"
      },
      "source": [
        "print(f'path to vocab file is : {vocab_file}') # will give us the vocab.txt file address\n",
        "print(f'Is LOWER CASE ? : {do_lower_case}') # will tell use whether to lower case or not "
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "path to vocab file is : b'/tmp/tfhub_modules/03d6fb3ce1605ad9e5e9ed5346b2fb9623ef4d3d/assets/vocab.txt'\n",
            "Is LOWER CASE ? : True\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vwWwe93VzGPx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# STEP 2\n",
        "FullTokenizer = bert.bert_tokenization.FullTokenizer\n",
        "tokenizer = FullTokenizer(vocab_file, do_lower_case)"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BNp8eC7txx47",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "outputId": "7229d7f0-7aa2-45f1-a7f4-409133ff5d14"
      },
      "source": [
        "# Tokinizer check \n",
        "tok_sent = tokenizer.tokenize('I love hiking!')\n",
        "print(f'Toenized Sentence - {tok_sent}')\n",
        "\n",
        "# How to convert to numbers? i.e. word2index counterpart of keras utility\n",
        "print(f\"Tokens to Index - {tokenizer.convert_tokens_to_ids(tokenizer.tokenize('I love hiking!'))}\")"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Toenized Sentence - ['i', 'love', 'hiking', '!']\n",
            "Tokens to Index - [1045, 2293, 13039, 999]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MNhcjPtKAi8n",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# BERT Style tokenization\n",
        "# Note that convert_tokens_to_ids is similar \n",
        "# to texts_to_sequences in tf.keras utility\n",
        "def encode_sentence(sent):\n",
        "    return tokenizer.convert_tokens_to_ids(tokenizer.tokenize(sent))"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dKW37BhFAnoh",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 449
        },
        "outputId": "03aa5c3b-48cb-49b3-f360-02af0441edc2"
      },
      "source": [
        "data_inputs = [encode_sentence(sentence) for sentence in data_clean]\n",
        "data_inputs[0]"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[22091,\n",
              " 2860,\n",
              " 2860,\n",
              " 2008,\n",
              " 1005,\n",
              " 1055,\n",
              " 1037,\n",
              " 26352,\n",
              " 5017,\n",
              " 1012,\n",
              " 2017,\n",
              " 2323,\n",
              " 2050,\n",
              " 2288,\n",
              " 2585,\n",
              " 12385,\n",
              " 1997,\n",
              " 2353,\n",
              " 2154,\n",
              " 2000,\n",
              " 2079,\n",
              " 2009,\n",
              " 1012,\n",
              " 1040]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uRqDcii1B6s6",
        "colab_type": "text"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iw1ldVccInVi",
        "colab_type": "text"
      },
      "source": [
        "### Data Creation\n",
        "\n",
        "\n",
        "                     Overview\n",
        "    \n",
        "Minimized padding strategy (Batch Padding)\n",
        "*   We will first pad the data, but we will  perform padding according to length of sentences present in each batch.\n",
        "*   We will Create batch in such a way that each batch will have sentences of similar length.\n",
        "\n",
        "*   With this strategy we will lose less data because of padding, because the paddings will be less.\n",
        "*   We will shuffle them so that same labeled data dont often appear in same batch.\n",
        "*   NOTE - We will have to do the training in batches, hence we could leverage padding per batch.\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cPDCHnv-JEI6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# [ \n",
        "    # [<'sent encoded'> , <'label'>, <'sent len'>]...\n",
        "# ]\n",
        "\n",
        "data_with_len = [[sent, data_labels[i], len(sent)]\n",
        "                 for i, sent in enumerate(data_inputs)]\n",
        "# shuffling data\n",
        "random.shuffle(data_with_len)\n",
        "# sort the data by sentences lengths \n",
        "data_with_len.sort(key=lambda x: x[2])\n",
        "\n",
        "# keep the data and label pair if the sentence length is greater than seven.\n",
        "# to get rid of smaller sentences.\n",
        "sorted_all = [(sent_lab[0], sent_lab[1])\n",
        "              for sent_lab in data_with_len if sent_lab[2] > 7]"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "75GS93doXEj2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "a321784a-eb53-4693-ad20-055029309232"
      },
      "source": [
        "sorted_all[0]"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "([5292, 2232, 2204, 2017, 2323, 3582, 2033, 1012], 1)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fp7SVxAhpGWq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "all_dataset = tf.data.Dataset.from_generator(lambda: sorted_all,\n",
        "                                             output_types=(tf.int32, tf.int32))"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MONg_85GpL3Q",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 73
        },
        "outputId": "3f77e993-d519-44fd-e750-206b735e8692"
      },
      "source": [
        "next(iter(all_dataset))"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(<tf.Tensor: shape=(8,), dtype=int32, numpy=array([5292, 2232, 2204, 2017, 2323, 3582, 2033, 1012], dtype=int32)>,\n",
              " <tf.Tensor: shape=(), dtype=int32, numpy=1>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jHobnzBwpPlo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# BATCHING\n",
        "BATCH_SIZE = 32\n",
        "all_batched = all_dataset.padded_batch(BATCH_SIZE, padded_shapes=((None, ), ()))"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8lwraDbFpbzr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "NB_BATCHES = math.ceil(len(sorted_all) / BATCH_SIZE)\n",
        "NB_BATCHES_TEST = NB_BATCHES // 10\n",
        "all_batched.shuffle(NB_BATCHES)\n",
        "test_dataset = all_batched.take(NB_BATCHES_TEST)\n",
        "train_dataset = all_batched.skip(NB_BATCHES_TEST)"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hpz3fQ9x2Z8p",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XUBL4VEIC0Q-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YXri2xD9C3nZ",
        "colab_type": "text"
      },
      "source": [
        "# III- Model Building\n",
        "\n",
        "Now that we have batches of our data ready, we can build any classification model over it. Let us see how to feed it to a CNN model(It can be replaced with LSTM/GRU too.\n",
        "\n",
        "Below is a 1-D Conv Net.\n",
        " - 6 feature detectors (We'll give more)\n",
        " - 3 different size of filters to detect different types of correlation between the words.\n",
        " - 2 feature map for each region sizeby max pooling.\n",
        " - We concatenate the outputs .\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "![image.png]()\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mIAQ1v6jC0lh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# MODEL class\n",
        "class DCNN(tf.keras.Model):\n",
        "    \n",
        "    def __init__(self,\n",
        "                 vocab_size,\n",
        "                 emb_dim=128,\n",
        "                 nb_filters=50,\n",
        "                 FFN_units=512,\n",
        "                 nb_classes=2,\n",
        "                 dropout_rate=0.1,\n",
        "                 training=False,\n",
        "                 name=\"dcnn\"):\n",
        "        super(DCNN, self).__init__(name=name)\n",
        "        \n",
        "        self.embedding = layers.Embedding(vocab_size,\n",
        "                                          emb_dim)\n",
        "        self.bigram = layers.Conv1D(filters=nb_filters,\n",
        "                                    kernel_size=2,\n",
        "                                    padding=\"valid\",\n",
        "                                    activation=\"relu\")\n",
        "        self.trigram = layers.Conv1D(filters=nb_filters,\n",
        "                                     kernel_size=3,\n",
        "                                     padding=\"valid\",\n",
        "                                     activation=\"relu\")\n",
        "        self.fourgram = layers.Conv1D(filters=nb_filters,\n",
        "                                      kernel_size=4,\n",
        "                                      padding=\"valid\",\n",
        "                                      activation=\"relu\")\n",
        "        self.pool = layers.GlobalMaxPool1D()\n",
        "        \n",
        "        self.dense_1 = layers.Dense(units=FFN_units, activation=\"relu\")\n",
        "        self.dropout = layers.Dropout(rate=dropout_rate)\n",
        "        if nb_classes == 2:\n",
        "            self.last_dense = layers.Dense(units=1,\n",
        "                                           activation=\"sigmoid\")\n",
        "        else:\n",
        "            self.last_dense = layers.Dense(units=nb_classes,\n",
        "                                           activation=\"softmax\")\n",
        "    \n",
        "    def call(self, inputs, training):\n",
        "        x = self.embedding(inputs)\n",
        "        x_1 = self.bigram(x) # batch_size, nb_filters, seq_len-1)\n",
        "        x_1 = self.pool(x_1) # (batch_size, nb_filters)\n",
        "        x_2 = self.trigram(x) # batch_size, nb_filters, seq_len-2)\n",
        "        x_2 = self.pool(x_2) # (batch_size, nb_filters)\n",
        "        x_3 = self.fourgram(x) # batch_size, nb_filters, seq_len-3)\n",
        "        x_3 = self.pool(x_3) # (batch_size, nb_filters)\n",
        "        \n",
        "        merged = tf.concat([x_1, x_2, x_3], axis=-1) # (batch_size, 3 * nb_filters)\n",
        "        merged = self.dense_1(merged)\n",
        "        merged = self.dropout(merged, training)\n",
        "        output = self.last_dense(merged)\n",
        "        \n",
        "        return output"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Odoz3LwDRDBi",
        "colab_type": "text"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g87-wFUWRJ8L",
        "colab_type": "text"
      },
      "source": [
        "# IV - Model Training\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RpXNyf6LRW7n",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "VOCAB_SIZE = len(tokenizer.vocab)\n",
        "EMB_DIM = 200\n",
        "NB_FILTERS = 100\n",
        "FFN_UNITS = 256\n",
        "NB_CLASSES = 2\n",
        "DROPOUT_RATE = 0.2\n",
        "NB_EPOCHS = 5"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-0BbLbqUSD0J",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "Dcnn = DCNN(vocab_size=VOCAB_SIZE,\n",
        "            emb_dim=EMB_DIM,\n",
        "            nb_filters=NB_FILTERS,\n",
        "            FFN_units=FFN_UNITS,\n",
        "            nb_classes=NB_CLASSES,\n",
        "            dropout_rate=DROPOUT_RATE)"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nMaWvGN5SN6g",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "if NB_CLASSES == 2:\n",
        "    Dcnn.compile(loss=\"binary_crossentropy\",\n",
        "                 optimizer=\"adam\",\n",
        "                 metrics=[\"accuracy\"])\n",
        "else:\n",
        "    Dcnn.compile(loss=\"sparse_categorical_crossentropy\",\n",
        "                 optimizer=\"adam\",\n",
        "                 metrics=[\"sparse_categorical_accuracy\"])"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U2Tp1qbxSQd-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "checkpoint_path = \"/content/drive/My Drive/DCNN_model/\"\n",
        "ckpt = tf.train.Checkpoint(Dcnn=Dcnn)\n",
        "ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=1)\n",
        "\n",
        "if ckpt_manager.latest_checkpoint:\n",
        "    ckpt.restore(ckpt_manager.latest_checkpoint)\n",
        "    print(\"Latest checkpoint restored!!\")"
      ],
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZSn9rkqnTyKl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class MyCustomCallback(tf.keras.callbacks.Callback):\n",
        "\n",
        "    def on_epoch_end(self, epoch, logs=None):\n",
        "        ckpt_manager.save()\n",
        "        print(\"Checkpoint saved at {}.\".format(checkpoint_path))"
      ],
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BKwuTJm-TsAZ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "outputId": "4d13a921-a0ff-41b0-a0b7-efd0c8dc01c8"
      },
      "source": [
        "Dcnn.fit(train_dataset,\n",
        "         epochs=NB_EPOCHS,\n",
        "         callbacks=[MyCustomCallback()])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/5\n",
            "  30720/Unknown - 2280s 74ms/step - loss: 0.3876 - accuracy: 0.8257"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SUPKbAOtZu-A",
        "colab_type": "text"
      },
      "source": [
        "# V- Model Evaluation & Prediction"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nIfFKwPLeky9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "results = Dcnn.evaluate(test_dataset)\n",
        "print(results)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cTjcj71Bexzr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_prediction(sentence):\n",
        "    tokens = encode_sentence(sentence)\n",
        "    inputs = tf.expand_dims(tokens, 0)\n",
        "\n",
        "    output = Dcnn(inputs, training=False)\n",
        "\n",
        "    sentiment = math.floor(output*2)\n",
        "\n",
        "    if sentiment == 0:\n",
        "        print(\"Ouput of the model: {}\\nPredicted sentiment: negative.\".format(\n",
        "            output))\n",
        "    elif sentiment == 1:\n",
        "        print(\"Ouput of the model: {}\\nPredicted sentiment: positive.\".format(\n",
        "            output))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Lts8eon_e547",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "get_prediction(\"This movie was pretty interesting.\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IlgPLJbXfGYz",
        "colab_type": "text"
      },
      "source": [
        "# Conclusion\n",
        "\n",
        "So we just saw the demo of using BERT Tokenizer utility. We tokenized and saw a mechanism to to minimize data loss through padding by doing batching.\n",
        "Note that goal of the above solution is to get an idea of how BERT tokenizer could be used and glued with model building. It can be any other model."
      ]
    }
  ]
}